import requests
import urllib.parse
from lxml import etree
# import chardet

baseUrl = 'https://www.bqkan8.com'

novelUrl = urllib.parse.urljoin(baseUrl, '/38_38836/')
headerVal = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56',

}

r = requests.get(novelUrl, timeout=10)
# html = etree.HTML(r.content.decode('gbk'))
html = etree.HTML(r.text)

hrefs = html.xpath('//div/dl/dd/a/@href')
print(hrefs)
# titles = html.xpath('//div/dl/dd/a/text()')
# gbk编码的！
# print(titles)
# code_type = chardet.detect(titles[0])

lists = {}
for i, href in enumerate(hrefs):
    chapterUrl = urllib.parse.urljoin(baseUrl, href)
    print(chapterUrl)
    # title = str(titles[i])
    # lists[title] = chapterUrl


# print(lists)
